package main

import (
	"fmt"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
)

func httpGet(url string) (result string, err error)  {
    r, err1:=http.Get(url)
    if err1!= nil{
    	err = err1
    	return
	}
    defer r.Body.Close()
    buf:= make([]byte,4*1024)
    for{
		n,_ := r.Body.Read(buf)
		if n == 0{
			break
		}
		result += string(buf[:n])
	}
    return
}
func getJoke(url string) (title,content string, err error)  {
	res,err1:=httpGet(url)
	if err1!=nil{
		err = err1
		return
	}
	re := regexp.MustCompile(`<h1>(?s:(.*?))</h1>`)
	if re== nil{
		fmt.Println("regexp.MustCompile:")
		return
	}
	contents := re.FindAllStringSubmatch(res,1)
	for _, data := range contents{
		title = data[1]
	}
	re1 := regexp.MustCompile(`<div class="content-txt pt10">(?s:(.*?))<a id="prev"`)
	if re1== nil{
		fmt.Println("regexp.MustCompile:")
		return
	}
	contents1 := re1.FindAllStringSubmatch(res,1)
	for _, data := range contents1{
		content = strings.TrimSpace(data[1])
	}
	return
}
func SaveToFile(i int,fileTitle,fileContent []string, page chan int){
	file, err := os.Create("joy_"+strconv.Itoa(i)+".txt")
	defer file.Close()
	if err!=nil{
		fmt.Println("os.Create Err:",err)
		return
	}
	defer file.Close()
	n:=len(fileTitle)
	for j:=0; j<n; j++{
		file.WriteString(fileTitle[j]+"\r\n"+fileContent[j]+"\r\n"+"======\r\n")
	}
	page <- i
	return
}
func spiderPage(i int, page chan int)  {
	url := "https://www.pengfu.com/xiaohua_"+ strconv.Itoa(i) +".html"
	fmt.Printf("正在爬取第%d页: %s\n",i,url)
	res,err:=httpGet(url)
	if err!=nil{
		fmt.Println("httpGet Err:", err)
		return
	}
	re := regexp.MustCompile(`<h1 class="dp-b"><a href="(?s:(.*?))"`)
	if re== nil{
		fmt.Println("regexp.MustCompile:")
		return
	}
	urls := re.FindAllStringSubmatch(res,-1)
	var fileTitle,fileContent []string
	for _,data := range urls{
		title,content,err1:=getJoke(data[1])
		if err1 != nil{
			fmt.Println("getJoke Error:",err1)
			return
		}
		fileTitle = append(fileTitle, title)
		fileContent = append(fileContent, content)
	}
	SaveToFile(i,fileTitle,fileContent,page)
}
func doWork(start,end int)  {
	page:=make(chan int)
    for i:=start; i<=end; i++{
    	go spiderPage(i,page)
	}
	for i:=start; i<=end; i++{
		fmt.Println("xxx")
		x:=<-page
    	fmt.Printf("第%d页爬取完毕\n",x)
		fmt.Println("yyy")
	}
}

func main()  {
	var start,end int
	fmt.Println("请输入开始的页码")
	fmt.Scan(&start)
	fmt.Println("请输入结束的页码")
	fmt.Scan(&end)
	doWork(start,end)
}
